﻿#light

module Dwarf_Managed

open System
open System.Collections.Generic
open System.Runtime.InteropServices
open MPI
open System.Threading

/// <summary>
/// Write the rules of command line structure.
/// </summary>
[<DllImport(@"CliTools.dll")>]
extern void usage();

let ROOT_PROCESS = 0

let getSettings(args : String[]) =
    if Communicator.world.Rank = ROOT_PROCESS then
        Dwarf_Managed_Common.Init(args) //Parse all arguments for dwarf.
            
    //Distribute size of matrices.
    MPI.Communicator.world.Broadcast(ref Dwarf_Managed_Common.rowNum, ROOT_PROCESS)

    //Init matrices.
    if Communicator.world.Rank <> ROOT_PROCESS then
        Dwarf_Managed_Common.value <- Array.zeroCreate<Double> Dwarf_Managed_Common.rowNum
      
        Dwarf_Managed_Common.col_ind <- Array.zeroCreate<Int32> Dwarf_Managed_Common.rowNum

        Dwarf_Managed_Common.vector <- Array.zeroCreate<Double> Dwarf_Managed_Common.rowNum

    MPI.Communicator.world.Broadcast<double[]>(ref Dwarf_Managed_Common.vector, ROOT_PROCESS)
    
    
let Solve() =
    //Loop for number of rows in matrix
    (*for i = 0 to Dwarf_Managed_Common.rowNum - 1 do
        //The C# compler/runtime is not
        //hoisting the array addressing out of the loop
        //hence the following makes a big difference
        //result[i] = 0;
        let mutable cell = 0.0
        //Loop for number of non-zero elements in current row
        for j = Dwarf_Managed_Common.row_ptr.[i] - 1 to Dwarf_Managed_Common.row_ptr.[i + 1] - 2 do
            //result[i] += vector[col_ind[j] - 1] * val[j];
            cell <- cell + Dwarf_Managed_Common.vector.[Dwarf_Managed_Common.col_ind.[j] - 1] * Dwarf_Managed_Common.value.[j]
        //see above comment       
        
        Dwarf_Managed_Common.result.[i] <- cell   *)
        
        
    let isRootProcess = Communicator.world.Rank = ROOT_PROCESS
    let numProcesses = Math.Min(Communicator.world.Size, Dwarf_Managed_Common.rowNum)

    let mutable valLocal = Array.zeroCreate<Double> 0
    let mutable col_indLocal = Array.zeroCreate<Int32> 0
    let mutable row_ptrLocal = Array.zeroCreate<Int32> 0
    
    let mutable numRowsOnProc = Array.zeroCreate<Int32> Communicator.world.Size
                        
    let mutable rowCount = Dwarf_Managed_Common.rowNum
    if isRootProcess then
        for i = 0 to numProcesses - 1 do
            numRowsOnProc.[i] <- Math.Min(Dwarf_Managed_Common.rowNum / numRowsOnProc.Length, rowCount)
            rowCount <- rowCount - numRowsOnProc.[i]
        
    Communicator.world.Broadcast<Int32[]>(ref numRowsOnProc, ROOT_PROCESS);
            
    let numRowsLocal = numRowsOnProc.[Communicator.world.Rank]        
    
    Communicator.world.ScatterFromFlattened(Dwarf_Managed_Common.row_ptr, numRowsOnProc, ROOT_PROCESS, ref row_ptrLocal)    
        
    let mutable lastRow_ptr = Array.zeroCreate<Int32> numProcesses
    let mutable lastRow_ptrLocal = Array.zeroCreate<Int32> 0
    if isRootProcess then
        let mutable lastPtr = 0
        for i = 0 to numProcesses - 1 do
            lastRow_ptr.[i] <- Dwarf_Managed_Common.row_ptr.[lastPtr + numRowsOnProc.[i]]
            lastPtr <- lastPtr + numRowsOnProc.[i]
            
    Communicator.world.ScatterFromFlattened(lastRow_ptr, 1, ROOT_PROCESS, ref lastRow_ptrLocal)
    
    Communicator.world.Barrier()
    
    let mutable numNonZeroElemsOnProc = Array.zeroCreate<Int32> Communicator.world.Size
    if isRootProcess then
        let mutable rowPtr = 0
        for i = 0 to numNonZeroElemsOnProc.Length - 1 do
            numNonZeroElemsOnProc.[i] <- Dwarf_Managed_Common.row_ptr.[rowPtr + numRowsOnProc.[i]] - Dwarf_Managed_Common.row_ptr.[rowPtr]
            rowPtr <- rowPtr + numRowsOnProc.[i]

    Communicator.world.Broadcast<Int32[]>(ref numNonZeroElemsOnProc, ROOT_PROCESS);
    
    col_indLocal <- Array.zeroCreate<Int32> numNonZeroElemsOnProc.[Communicator.world.Rank]
    valLocal <- Array.zeroCreate<Double> numNonZeroElemsOnProc.[Communicator.world.Rank]
    
    Communicator.world.ScatterFromFlattened(Dwarf_Managed_Common.col_ind, numNonZeroElemsOnProc, ROOT_PROCESS, ref col_indLocal)
    Communicator.world.ScatterFromFlattened(Dwarf_Managed_Common.value, numNonZeroElemsOnProc, ROOT_PROCESS, ref valLocal)

    let mutable tempResult = Array.zeroCreate<Double> numRowsLocal
    //Loop for number of rows in matrix
    for i = 0 to numRowsLocal - 1 do
        //The C# compler/runtime is not
        //hoisting the array addressing out of the loop
        //hence the following makes a big difference
        //result[i] = 0;
        let mutable cell = 0.0
        let ptr =             
            if i = numRowsLocal - 1 then
                lastRow_ptrLocal.[0] - row_ptrLocal.[0]
            else
                row_ptrLocal.[i + 1] - row_ptrLocal.[0]
                
        //var ptr = (i == numRowsLocal - 1 ? lastRow_ptrLocal[0] : row_ptrLocal[i + 1]) - row_ptrLocal[0];
                        
        //Loop for number of non-zero elements in current row
        for j = row_ptrLocal.[i] - row_ptrLocal.[0] to ptr - 1 do
            cell <- cell + Dwarf_Managed_Common.vector.[col_indLocal.[j] - 1] * valLocal.[j]
        //see above comment
        tempResult.[i] <- cell

    Dwarf_Managed_Common.result = Communicator.world.GatherFlattened(tempResult, ROOT_PROCESS);
 
let main() =   
    
    try 
        let args = System.Environment.GetCommandLineArgs()
        let env = new MPI.Environment(ref args)             // MPI Initialization.
        if Communicator.world.Size = 1 then
            raise (new Exception("Only one mpi process."))

        getSettings(args)                                   // Get settings from the command line.

        Communicator.world.Barrier()
        if Communicator.world.Rank = ROOT_PROCESS then
            Dwarf_Managed_Common.Start()                    // Start new time count.
        for i = 0 to 200 do
            Solve()                                         // Solve the current problem.
        Communicator.world.Barrier()
        if Communicator.world.Rank = ROOT_PROCESS then
            Dwarf_Managed_Common.Finish()                   // Stop the time count and write results.
        
        env.Dispose()
    with e -> Console.WriteLine(e.Message); Console.WriteLine(e.StackTrace); usage()
         
main()
